{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "\n",
    "# Paths (adjust if needed)\n",
    "eval_dir = \"/home/jianingy/research/fast3r/notebooks/RealEstate10K_eval\"\n",
    "test_file = \"/home/jianingy/research/fast3r/notebooks/re10k_test_1800.txt\"\n",
    "\n",
    "# Read the list of scene IDs into a set for fast lookup\n",
    "with open(test_file, \"r\") as f:\n",
    "    test_scenes = set(line.strip() for line in f if line.strip())\n",
    "\n",
    "# Metrics we want to average\n",
    "metrics = [\"RRA_at_5\", \"RRA_at_15\", \"RRA_at_30\",\n",
    "           \"RTA_at_5\", \"RTA_at_15\", \"RTA_at_30\",\n",
    "           \"mAA_30\"]\n",
    "\n",
    "# Initialize accumulators\n",
    "acc = {m: 0.0 for m in metrics}\n",
    "count = 0\n",
    "\n",
    "# Go through each file in the eval_dir\n",
    "for filename in os.listdir(eval_dir):\n",
    "    if filename.endswith(\".txt\"):\n",
    "        filepath = os.path.join(eval_dir, filename)\n",
    "\n",
    "        # Read JSON text.  If it's single-quoted JSON, handle with `json.loads` carefully\n",
    "        # or use `ast.literal_eval`. If your data uses proper double-quoted JSON, regular json.loads will do.\n",
    "        with open(filepath, \"r\") as ff:\n",
    "            # If your data is strictly valid JSON, you can do:\n",
    "            # data = json.load(ff)\n",
    "            # If it has single quotes or other slight deviations, you can do:\n",
    "            text = ff.read()\n",
    "            # Convert single quotes to double quotes if your data is actually single-quoted\n",
    "            text = text.replace(\"'\", \"\\\"\")\n",
    "            data = json.loads(text)\n",
    "\n",
    "        video_name = data.get(\"video_name\")\n",
    "\n",
    "        # Check if this video is in our test set\n",
    "        if video_name in test_scenes:\n",
    "            # Accumulate each metric\n",
    "            for m in metrics:\n",
    "                acc[m] += data.get(m, 0.0)\n",
    "            count += 1\n",
    "\n",
    "# Compute averages if we found matches\n",
    "if count > 0:\n",
    "    for m in metrics:\n",
    "        acc[m] /= count\n",
    "\n",
    "print(f\"Number of matched scenes: {count}\")\n",
    "print(\"Averaged metrics across matched scenes:\")\n",
    "for m in metrics:\n",
    "    print(f\"{m}: {acc[m]:.6f}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# find the worst 10 scenes on mAA_30\n",
    "worst_10 = []\n",
    "for filename in os.listdir(eval_dir):\n",
    "    if filename.endswith(\".txt\"):\n",
    "        filepath = os.path.join(eval_dir, filename)\n",
    "\n",
    "        # Read JSON text.  If it's single-quoted JSON, handle with `json.loads` carefully\n",
    "        # or use `ast.literal_eval`. If your data uses proper double-quoted JSON, regular json.loads will do.\n",
    "        with open(filepath, \"r\") as ff:\n",
    "            # If your data is strictly valid JSON, you can do:\n",
    "            # data = json.load(ff)\n",
    "            # If it has single quotes or other slight deviations, you can do:\n",
    "            text = ff.read()\n",
    "            # Convert single quotes to double quotes if your data is actually single-quoted\n",
    "            text = text.replace(\"'\", \"\\\"\")\n",
    "            data = json.loads(text)\n",
    "\n",
    "        video_name = data.get(\"video_name\")\n",
    "\n",
    "        # Check if this video is in our test set\n",
    "        if video_name in test_scenes:\n",
    "            worst_10.append((video_name, data.get(\"mAA_30\", 0.0)))\n",
    "\n",
    "print(\"Worst 10 scenes on mAA_30:\")\n",
    "worst_10.sort(key=lambda x: x[1])\n",
    "for video_name, score in worst_10[:10]:\n",
    "    print(f\"{video_name}: {score:.6f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "fast3r",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
